# -*- coding: UTF-8 -*-
# @file:bilibili_yonghu.py


import requests
from lxml import etree
import time
from pymongo import MongoClient
import json

client = MongoClient()
collection = client["bilibili"]["user_data"]


headers = {
    'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Mobile Safari/537.36',
    'cookie': "CURRENT_FNVAL=80; _uuid=CB720616-E5F6-BF79-38BC-0F30CFB8C1A747362infoc; blackside_state=1; rpdid=|(mm~|lRRmk0J'uY|lum~uR); CURRENT_QUALITY=116; fingerprint3=eb0afaa87b90c36a8487347a7f1c45cf; buvid_fp_plain=6252505F-8D07-4D08-9A1C-8AB170A9B68D143076infoc; buivd_fp=6252505F-8D07-4D08-9A1C-8AB170A9B68D143076infoc; LIVE_BUVID=AUTO5816090469446518; fingerprint_s=567717c5ab28c80e6b4c59a9b2a2139a; buvid_fp=6252505F-8D07-4D08-9A1C-8AB170A9B68D143076infoc; fingerprint=0613ef7eef748f542fcece2b25641f9b; SESSDATA=57947e16%2C1631942547%2Cec70b%2A31; bili_jct=81700d4c2316f2005837059261d6308e; DedeUserID=298746809; DedeUserID__ckMd5=64c6bf5e91e0cf4b; sid=8pvy251l; buvid3=AFE98866-57DF-4D5D-81AD-DD9B622B0DEE18532infoc; bsource=search_baidu; PVID=1; finger=1571944565"

}

item={}


#获取用户名字和认证
def crawl(url):
    html = requests.get(url,headers=headers)
    pars = etree.HTML(html.text)



    item['user_name'] = pars.xpath('//*[@id="app"]/div/div[2]/div[3]/div[1]/span[1]/text()')
    if len(item['user_name']) == 0:
        item['user_name'] = 'pass'
    item['verify'] = pars.xpath('//div[@class="verify"]/span/text()')
    if len(item['verify']) == 0:
        item['verify'] = 'None'
    return crawl_json



#获取粉丝和关注

def crawl_json(url2):
    html = requests.get(url2, headers=headers)
    parse_html = html.text
    data = json.loads(parse_html)['data']

    item['following'] = data['following']
    item['follower'] = data['follower']

    print(item)
    collection.insert_one(dict(item))



#range控制需要爬取的用户的范围  从1开始 目前按最大值为652672711，可以自定义需要爬取的范围
for i in range(1,200):
    url = 'https://m.bilibili.com/space/{}'.format(i)
    url2 = 'https://api.bilibili.com/x/relation/stat?vmid={}'.format(i)  #获取json里面的粉丝和关注数
    crawl(url)
    crawl_json(url2)
    time.sleep(0.3)



